package org.apache.lucene.analysis.jjs;

import java.io.IOException;
import java.io.Reader;
import java.util.LinkedList;
import java.util.Queue;
import java.util.logging.Level;

import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.CharUtil;

import ckip.utils.CKIPSeg;

import com.chenlb.mmseg4j.Word;

public class CKIPSegTokenizer extends Tokenizer{
	private Queue<Word> wqueue = new LinkedList<Word>();
	private int curPos=0;
	private TermAttribute termAtt;
	private OffsetAttribute offsetAtt;
	private TypeAttribute typeAtt;
	private CKIPSeg seg;

	public CKIPSegTokenizer(Reader input)
	{
		super(input);
		
		termAtt = (TermAttribute)addAttribute(TermAttribute.class);
		offsetAtt = (OffsetAttribute)addAttribute(OffsetAttribute.class);
		typeAtt = (TypeAttribute)addAttribute(TypeAttribute.class);
		
		String 	account="john"; 	// The registered account
		String 	passwd="john7810"; 	// The password of registered account
		String 	ckipHostname=null; 	// null will use default built-in CKIP hostname;
		int		ckipPort=-1; 		// -1 will use default port.
		Level	logLevel=Level.WARNING;
		seg = new CKIPSeg(account, passwd, ckipHostname, ckipPort, logLevel);
	}

	@Override
	public void reset(Reader input) throws IOException {
		super.reset(input);		
		curPos=0;
		wqueue.clear();
	}
	
	protected void ckipSeg(String sent) throws IOException
	{
		try
		{
			int cp=curPos;
			for(String term:seg.querySegment(sent))
			{
				wqueue.add(new Word(term.toCharArray(), cp));
				cp+=term.length();
			}
			curPos=cp;
		}
		catch(Exception e){throw new IOException(e.toString());}	
	}
	
	@Override
	public boolean incrementToken() throws IOException {
		clearAttributes();
		if(wqueue.isEmpty())
		{
			StringBuffer sentBuf = new StringBuffer();
			int c;
			boolean isSeg=false;
			while((c=input.read())!=-1)
			{
				//System.out.printf("\t[Test] Read '%d'...\n", c);
				if(CharUtil.isChinese((char)c))
				{
					sentBuf.append((char)c);
				}
				else
				{
					if(sentBuf.length()>0)
					{
						ckipSeg(sentBuf.toString());	
						isSeg=true;
						break;
					}
					else
					{
						curPos++;
					}
				}				
			} // while((c=(char)input.read())!=-1)		
			if(!isSeg && sentBuf.length()>0) ckipSeg(sentBuf.toString());
		}
		Word word = wqueue.poll();
		if(word!=null)
		{
			termAtt.setTermBuffer(word.getSen(), word.getWordOffset(), word.getLength());
			offsetAtt.setOffset(word.getStartOffset(), word.getEndOffset());
			typeAtt.setType(word.getType());
			return true;
		}
		end();
		return false;
	}

}
